package com.rjt.spider;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.proxy.SimpleProxyProvider;

public class BaymSpider implements PageProcessor{

	private Site site = Site.me().setRetryTimes(3).setSleepTime(100).setTimeOut(100000);
	
	

    @Override
    public void process(Page page) {
    	
    	 page.addTargetRequests(page.getHtml().links().regex("(http://8ym.cn/rptlist/[\\w\\-]+)").all());
    	 page.putField("url", page.getHtml().$(".list-group-item").regex("<a href=\"(.*?)\" class=\"list-group-item\"><span class=\"badge (.*?)\">(.*?)</span>(.*?)</a>",1).all());
    	 page.putField("date", page.getHtml().$(".list-group-item").regex("<a href=\"(.*?)\" class=\"list-group-item\"><span class=\"badge (.*?)\">(.*?)</span>(.*?)</a>",3).all());
    	 page.putField("title", page.getHtml().$(".list-group-item").regex("<a href=\"(.*?)\" class=\"list-group-item\"><span class=\"badge (.*?)\">(.*?)</span>(.*?)</a>",4).all());

    }

    @Override
    public Site getSite() {
    	
        return site;
    }

    public static void main(String[] args) {
    	
    	HttpClientDownloader httpClientDownloader=new HttpClientDownloader();
    	httpClientDownloader.setProxyProvider(SimpleProxyProvider.from(new Proxy("127.0.0.1", 8888)));
        Spider.create(new BaymSpider()).addPipeline(new FilePipeline("test.json")).addUrl("http://8ym.cn/rptlist").setDownloader(httpClientDownloader).thread(1).run();
    	
    }
}
